#!/usr/bin/python

import json
import sys
import numpy as np
import matplotlib.pyplot as plt
import collections
import ast
import pickle
import multiprocessing
import os
from time import sleep
from time import time
from itertools import groupby
from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import confusion_matrix
from sklearn.datasets import make_multilabel_classification
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.multiclass import OneVsOneClassifier
from sklearn import datasets
from sklearn.externals import joblib



validLabels = None
StatFile = None
CMFile = None
PHFile = None
PHTable = None
LabelDict = None
text_clf = None

def main():	
	# Not Run Yet
	test(1,'PER_ascii_stem_stopword','Label_Leaves',Beam,'../../../Data229/CLF/NB_1/');

def test(trainThres,data,label,method,path):
	testName=method.__name__
	
	print '--------------------------'
	print 'trainThres = ',trainThres
	print 'testName = ',testName
	print 'data = ',data
	print 'label = ',label

	dataFile = open('../'+data,'r')
	labelFile = open('../'+label,'r')

	#StatFile = open('../output/Stats_'+modelName+'_'+str(trainThres)+'_'+data+'_'+label+'.txt','w')
	
	global PHFile, PHTable
	PHFile = open('../ParentHash.json','r')
	PHTable = json.load(PHFile)

	global LabelDict	
	LabelDict = dict()
	for l in range(1,10):
		LabelDict[l] = np.loadtxt('../validLabels_L'+str(l)+'.txt',dtype=str)

	global validLabels
	validLabels = np.loadtxt('../validLabels_L1.txt',dtype=str)

	trainData = []
	trainLabel = []
	testData = []
	testLabel = []
	
	cnt=0
	for line in dataFile:
		#print 'Read Data #',cnt+1
		if (cnt % 10) == 9:
			testData.append(line.strip())
		elif (cnt % 10) < trainThres:
			trainData.append(line.strip())
		cnt += 1
	print 'Finish Reading Data'

	cnt=0
	for line in labelFile:
		#print 'Read Label #',cnt+1
		if (cnt % 10) == 9:
			testLabel.append(line.strip())
		elif (cnt % 10) < trainThres:
			trainLabel.append(line.strip())
		cnt += 1
	print 'Finish Reading Labels'

	print 'Train Data Size = ',len(trainData)
	print 'Train Label Size = ',len(trainLabel)
	print 'Test Data Size = ',len(testData)
	print 'Test Label Size = ',len(testLabel)
	print ''

	LoadCLF(trainData,trainLabel,path)
	predictedTrain, predictedTest = method(trainData,trainLabel,testData,testLabel)

	np.savetxt('../output/Predicted_'+testName+'_'+str(trainThres)+'_'+data+'_'+label+'_Train_.txt',np.array(predictedTrain), fmt='%d')
	np.savetxt('../output/Predicted_'+testName+'_'+str(trainThres)+'_'+data+'_'+label+'_Test_.txt',np.array(predictedTest), fmt='%d')

	
def LoadCLF_thread(clf,path):
	global text_clf
	if os.path.isfile(path+clf+'.pkl'):
		text_clf[clf]=joblib.load(path+clf+'.pkl')
		print text_clf[clf]
	
def LoadCLF(trainData,trainLabels,path):

	#Train
	time0 = time()
	main_thread = multiprocessing.current_process()

	global text_clf
	text_clf = dict()
	text_clf['0'] = None

	for i in trainLabels:
		curr = i
		while curr != '0' :
			if (PHTable[curr] not in text_clf): # and (PHTable[curr] in LabelDict[1]) and False:
				text_clf[PHTable[curr]]=None
			curr = PHTable[curr]
			
	print 'CLFs allocated'
	CLFsize = len(text_clf)
	print 'CLFs Size = ',CLFsize

	cnt=0
	for clf in text_clf:
		cnt+=1
		print 'Loading CLF',cnt,'/',CLFsize, '(',clf,')','time =',(time()-time0)
		if os.path.isfile(path+clf+'.pkl'):
			text_clf[clf]=joblib.load(path+clf+'.pkl')
		
		#while True:
			#if len(multiprocessing.active_children()) < 8:
				#multiprocessing.Process(target=LoadCLF_thread, args=(clf,path)).start()
				#break
			#else:
				#sleep(0.1)

	#while len(multiprocessing.active_children()) != 0:
		#print 'wait for subprocess to finish'
		#sleep(1)
		
	print 'Loading is Done'

def Predict(data,idx,predicted):
	curr='0'
	while (curr in text_clf) and text_clf[curr] != None:
		curr = text_clf[curr].predict([data])[0]
	predicted[idx] = curr

def Beam(trainData,trainLabels,testData,testLabels):

	CLFsize = len(text_clf)

	time0 = time()
	
	testSize = len(testData)
	predictedTest = [None] * testSize
	cnt = 0
	for t in testData:
		cnt+=1
		print 'Predicting Test',cnt,'/',testSize, 'time =',(time()-time0)

		while True:
			if len(multiprocessing.active_children()) < 8:
				multiprocessing.Process(target=Predict, args=(t,cnt-1,predictedTest)).start()
				break
			else:
				sleep(0.1)
	
	trainSize = len(trainData)
	predictedTrain = [None] * trainSize
	cnt = 0
	for t in trainData:
		cnt+=1
		print 'Predicting Train',cnt,'/',trainSize, 'time =',(time()-time0)

		while True:
			if len(multiprocessing.active_children()) < 8:
				multiprocessing.Process(target=Predict, args=(t,cnt-1,predictedTrain)).start()
				break
			else:
				sleep(0.1)

	while len(multiprocessing.active_children()) != 0:
		print 'wait for subprocess to finish'
		sleep(1)

	return predictedTrain,predictedTest

def printLog(f,s):
	print s
	print >>f, s
	
if __name__ == "__main__":
    main()
